{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `vaex` @ EuroPython 2019\n",
    "## New York Taxi Dataset (2009-2015): Exploratory Data Analysis and Machine Learning example (part 3/3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-10T20:41:52.313531Z",
     "start_time": "2019-07-10T20:41:50.434304Z"
    }
   },
   "outputs": [],
   "source": [
    "import vaex"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read in the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-10T20:41:52.359144Z",
     "start_time": "2019-07-10T20:41:52.315339Z"
    }
   },
   "outputs": [],
   "source": [
    "df = vaex.open('./data/yellow_taxi_2009_2015_f32.hdf5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Re-define the test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-10T20:41:52.365599Z",
     "start_time": "2019-07-10T20:41:52.361093Z"
    }
   },
   "outputs": [],
   "source": [
    "# The test set\n",
    "df_test = df[1_026_944_937:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inspect the test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-10T20:41:52.403650Z",
     "start_time": "2019-07-10T20:41:52.367734Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                            </th><th>vendor_id  </th><th>pickup_datetime              </th><th>dropoff_datetime             </th><th style=\"text-align: right;\">  passenger_count</th><th>payment_type  </th><th style=\"text-align: right;\">  trip_distance</th><th style=\"text-align: right;\">  pickup_longitude</th><th style=\"text-align: right;\">  pickup_latitude</th><th style=\"text-align: right;\">  rate_code</th><th style=\"text-align: right;\">  store_and_fwd_flag</th><th style=\"text-align: right;\">  dropoff_longitude</th><th style=\"text-align: right;\">  dropoff_latitude</th><th style=\"text-align: right;\">  fare_amount</th><th style=\"text-align: right;\">  surcharge</th><th style=\"text-align: right;\">  mta_tax</th><th style=\"text-align: right;\">  tip_amount</th><th style=\"text-align: right;\">  tolls_amount</th><th style=\"text-align: right;\">  total_amount</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i></td><td>VTS        </td><td>2014-12-15 18:23:00.000000000</td><td>2014-12-15 18:58:00.000000000</td><td style=\"text-align: right;\">                2</td><td>              </td><td style=\"text-align: right;\">           6.28</td><td style=\"text-align: right;\">          -74.0042</td><td style=\"text-align: right;\">          40.7212</td><td style=\"text-align: right;\">          1</td><td style=\"text-align: right;\">                 nan</td><td style=\"text-align: right;\">           -73.97  </td><td style=\"text-align: right;\">          nan     </td><td style=\"text-align: right;\">        nan  </td><td style=\"text-align: right;\">      nan  </td><td style=\"text-align: right;\">    nan  </td><td style=\"text-align: right;\">      nan   </td><td style=\"text-align: right;\">        nan   </td><td style=\"text-align: right;\">        nan   </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i></td><td>VTS        </td><td>2015-01-15 19:05:39.000000000</td><td>2015-01-15 19:23:42.000000000</td><td style=\"text-align: right;\">                1</td><td>1             </td><td style=\"text-align: right;\">           1.59</td><td style=\"text-align: right;\">          -73.9939</td><td style=\"text-align: right;\">          40.7501</td><td style=\"text-align: right;\">          1</td><td style=\"text-align: right;\">                   0</td><td style=\"text-align: right;\">           -73.9748</td><td style=\"text-align: right;\">           40.7506</td><td style=\"text-align: right;\">         12  </td><td style=\"text-align: right;\">        1  </td><td style=\"text-align: right;\">      0.5</td><td style=\"text-align: right;\">        3.25</td><td style=\"text-align: right;\">          0   </td><td style=\"text-align: right;\">         17.05</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i></td><td>CMT        </td><td>2015-01-10 20:33:38.000000000</td><td>2015-01-10 20:53:28.000000000</td><td style=\"text-align: right;\">                1</td><td>1             </td><td style=\"text-align: right;\">           3.3 </td><td style=\"text-align: right;\">          -74.0016</td><td style=\"text-align: right;\">          40.7242</td><td style=\"text-align: right;\">          1</td><td style=\"text-align: right;\">                   0</td><td style=\"text-align: right;\">           -73.9944</td><td style=\"text-align: right;\">           40.7591</td><td style=\"text-align: right;\">         14.5</td><td style=\"text-align: right;\">        0.5</td><td style=\"text-align: right;\">      0.5</td><td style=\"text-align: right;\">        2   </td><td style=\"text-align: right;\">          0   </td><td style=\"text-align: right;\">         17.8 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i></td><td>CMT        </td><td>2015-01-10 20:33:38.000000000</td><td>2015-01-10 20:43:41.000000000</td><td style=\"text-align: right;\">                1</td><td>2             </td><td style=\"text-align: right;\">           1.8 </td><td style=\"text-align: right;\">          -73.9633</td><td style=\"text-align: right;\">          40.8028</td><td style=\"text-align: right;\">          1</td><td style=\"text-align: right;\">                   0</td><td style=\"text-align: right;\">           -73.9518</td><td style=\"text-align: right;\">           40.8244</td><td style=\"text-align: right;\">          9.5</td><td style=\"text-align: right;\">        0.5</td><td style=\"text-align: right;\">      0.5</td><td style=\"text-align: right;\">        0   </td><td style=\"text-align: right;\">          0   </td><td style=\"text-align: right;\">         10.8 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i></td><td>CMT        </td><td>2015-01-10 20:33:39.000000000</td><td>2015-01-10 20:35:31.000000000</td><td style=\"text-align: right;\">                1</td><td>2             </td><td style=\"text-align: right;\">           0.5 </td><td style=\"text-align: right;\">          -74.0091</td><td style=\"text-align: right;\">          40.7138</td><td style=\"text-align: right;\">          1</td><td style=\"text-align: right;\">                   0</td><td style=\"text-align: right;\">           -74.0043</td><td style=\"text-align: right;\">           40.72  </td><td style=\"text-align: right;\">          3.5</td><td style=\"text-align: right;\">        0.5</td><td style=\"text-align: right;\">      0.5</td><td style=\"text-align: right;\">        0   </td><td style=\"text-align: right;\">          0   </td><td style=\"text-align: right;\">          4.8 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>5</i></td><td>CMT        </td><td>2015-01-10 20:33:39.000000000</td><td>2015-01-10 20:52:58.000000000</td><td style=\"text-align: right;\">                1</td><td>2             </td><td style=\"text-align: right;\">           3   </td><td style=\"text-align: right;\">          -73.9712</td><td style=\"text-align: right;\">          40.7624</td><td style=\"text-align: right;\">          1</td><td style=\"text-align: right;\">                   0</td><td style=\"text-align: right;\">           -74.0042</td><td style=\"text-align: right;\">           40.7427</td><td style=\"text-align: right;\">         15  </td><td style=\"text-align: right;\">        0.5</td><td style=\"text-align: right;\">      0.5</td><td style=\"text-align: right;\">        0   </td><td style=\"text-align: right;\">          0   </td><td style=\"text-align: right;\">         16.3 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>6</i></td><td>CMT        </td><td>2015-01-10 20:33:39.000000000</td><td>2015-01-10 20:53:52.000000000</td><td style=\"text-align: right;\">                1</td><td>1             </td><td style=\"text-align: right;\">           9   </td><td style=\"text-align: right;\">          -73.8744</td><td style=\"text-align: right;\">          40.774 </td><td style=\"text-align: right;\">          1</td><td style=\"text-align: right;\">                   0</td><td style=\"text-align: right;\">           -73.987 </td><td style=\"text-align: right;\">           40.7582</td><td style=\"text-align: right;\">         27  </td><td style=\"text-align: right;\">        0.5</td><td style=\"text-align: right;\">      0.5</td><td style=\"text-align: right;\">        6.7 </td><td style=\"text-align: right;\">          5.33</td><td style=\"text-align: right;\">         40.33</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>7</i></td><td>CMT        </td><td>2015-01-10 20:33:39.000000000</td><td>2015-01-10 20:58:31.000000000</td><td style=\"text-align: right;\">                1</td><td>2             </td><td style=\"text-align: right;\">           2.2 </td><td style=\"text-align: right;\">          -73.9833</td><td style=\"text-align: right;\">          40.726 </td><td style=\"text-align: right;\">          1</td><td style=\"text-align: right;\">                   0</td><td style=\"text-align: right;\">           -73.9925</td><td style=\"text-align: right;\">           40.7496</td><td style=\"text-align: right;\">         14  </td><td style=\"text-align: right;\">        0.5</td><td style=\"text-align: right;\">      0.5</td><td style=\"text-align: right;\">        0   </td><td style=\"text-align: right;\">          0   </td><td style=\"text-align: right;\">         15.3 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>8</i></td><td>CMT        </td><td>2015-01-10 20:33:39.000000000</td><td>2015-01-10 20:42:20.000000000</td><td style=\"text-align: right;\">                3</td><td>1             </td><td style=\"text-align: right;\">           0.8 </td><td style=\"text-align: right;\">          -74.0027</td><td style=\"text-align: right;\">          40.7341</td><td style=\"text-align: right;\">          1</td><td style=\"text-align: right;\">                   0</td><td style=\"text-align: right;\">           -73.995 </td><td style=\"text-align: right;\">           40.7263</td><td style=\"text-align: right;\">          7  </td><td style=\"text-align: right;\">        0.5</td><td style=\"text-align: right;\">      0.5</td><td style=\"text-align: right;\">        1.66</td><td style=\"text-align: right;\">          0   </td><td style=\"text-align: right;\">          9.96</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>9</i></td><td>CMT        </td><td>2015-01-10 20:33:39.000000000</td><td>2015-01-10 21:11:35.000000000</td><td style=\"text-align: right;\">                3</td><td>2             </td><td style=\"text-align: right;\">          18.2 </td><td style=\"text-align: right;\">          -73.783 </td><td style=\"text-align: right;\">          40.6444</td><td style=\"text-align: right;\">          2</td><td style=\"text-align: right;\">                   0</td><td style=\"text-align: right;\">           -73.9876</td><td style=\"text-align: right;\">           40.7594</td><td style=\"text-align: right;\">         52  </td><td style=\"text-align: right;\">        0  </td><td style=\"text-align: right;\">      0.5</td><td style=\"text-align: right;\">        0   </td><td style=\"text-align: right;\">          5.33</td><td style=\"text-align: right;\">         58.13</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "  #  vendor_id    pickup_datetime                dropoff_datetime                 passenger_count  payment_type      trip_distance    pickup_longitude    pickup_latitude    rate_code    store_and_fwd_flag    dropoff_longitude    dropoff_latitude    fare_amount    surcharge    mta_tax    tip_amount    tolls_amount    total_amount\n",
       "  0  VTS          2014-12-15 18:23:00.000000000  2014-12-15 18:58:00.000000000                  2                             6.28            -74.0042            40.7212            1                   nan             -73.97              nan               nan          nan        nan          nan             nan             nan\n",
       "  1  VTS          2015-01-15 19:05:39.000000000  2015-01-15 19:23:42.000000000                  1  1                          1.59            -73.9939            40.7501            1                     0             -73.9748             40.7506           12            1          0.5          3.25            0              17.05\n",
       "  2  CMT          2015-01-10 20:33:38.000000000  2015-01-10 20:53:28.000000000                  1  1                          3.3             -74.0016            40.7242            1                     0             -73.9944             40.7591           14.5          0.5        0.5          2               0              17.8\n",
       "  3  CMT          2015-01-10 20:33:38.000000000  2015-01-10 20:43:41.000000000                  1  2                          1.8             -73.9633            40.8028            1                     0             -73.9518             40.8244            9.5          0.5        0.5          0               0              10.8\n",
       "  4  CMT          2015-01-10 20:33:39.000000000  2015-01-10 20:35:31.000000000                  1  2                          0.5             -74.0091            40.7138            1                     0             -74.0043             40.72              3.5          0.5        0.5          0               0               4.8\n",
       "  5  CMT          2015-01-10 20:33:39.000000000  2015-01-10 20:52:58.000000000                  1  2                          3               -73.9712            40.7624            1                     0             -74.0042             40.7427           15            0.5        0.5          0               0              16.3\n",
       "  6  CMT          2015-01-10 20:33:39.000000000  2015-01-10 20:53:52.000000000                  1  1                          9               -73.8744            40.774             1                     0             -73.987              40.7582           27            0.5        0.5          6.7             5.33           40.33\n",
       "  7  CMT          2015-01-10 20:33:39.000000000  2015-01-10 20:58:31.000000000                  1  2                          2.2             -73.9833            40.726             1                     0             -73.9925             40.7496           14            0.5        0.5          0               0              15.3\n",
       "  8  CMT          2015-01-10 20:33:39.000000000  2015-01-10 20:42:20.000000000                  3  1                          0.8             -74.0027            40.7341            1                     0             -73.995              40.7263            7            0.5        0.5          1.66            0               9.96\n",
       "  9  CMT          2015-01-10 20:33:39.000000000  2015-01-10 21:11:35.000000000                  3  2                         18.2             -73.783             40.6444            2                     0             -73.9876             40.7594           52            0          0.5          0               5.33           58.13"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# View the head of the DataFrame\n",
    "df_test.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the _state_ into the test DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-10T20:41:54.448454Z",
     "start_time": "2019-07-10T20:41:52.407107Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jovan/miniconda3/lib/python3.7/site-packages/lightgbm/__init__.py:46: UserWarning: Starting from version 2.2.1, the library file in distribution wheels for macOS is built by the Apple Clang (Xcode_9.4.1) compiler.\n",
      "This means that in case of installing LightGBM from PyPI via the ``pip install lightgbm`` command, you don't need to install the gcc compiler anymore.\n",
      "Instead of that, you need to install the OpenMP library, which is required for running LightGBM on the system with the Apple Clang compiler.\n",
      "You can install the OpenMP library by the following command: ``brew install libomp``.\n",
      "  \"You can install the OpenMP library by the following command: ``brew install libomp``.\", UserWarning)\n"
     ]
    }
   ],
   "source": [
    "# This is where the magic happens\n",
    "df_test.state_load('./data/taxi_ml_state.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-10T20:42:03.283341Z",
     "start_time": "2019-07-10T20:41:54.450238Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<string>:1: RuntimeWarning: invalid value encountered in greater\n",
      "<string>:1: RuntimeWarning: invalid value encountered in less\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                             </th><th>vendor_id  </th><th>pickup_datetime              </th><th>dropoff_datetime             </th><th>passenger_count  </th><th>payment_type  </th><th>trip_distance     </th><th>pickup_longitude  </th><th>pickup_latitude   </th><th>rate_code  </th><th>store_and_fwd_flag  </th><th>dropoff_longitude  </th><th>dropoff_latitude  </th><th>fare_amount  </th><th>surcharge  </th><th>mta_tax  </th><th>tip_amount        </th><th>tolls_amount     </th><th>total_amount      </th><th>trip_speed_mph    </th><th>trip_duration_min  </th><th>fare_by_distance  </th><th>pu_hour  </th><th>pu_day_of_week  </th><th>pu_month  </th><th>pu_is_weekend  </th><th>arc_distance_miles  </th><th>direction_angle    </th><th>PCA_0                 </th><th>PCA_1                 </th><th>PCA_2                 </th><th>PCA_3                 </th><th>payment_type_  </th><th>lightgbm_prediction  </th><th>onehot_payment_type__1  </th><th>onehot_payment_type__2  </th><th>onehot_payment_type__3  </th><th>onehot_payment_type__4  </th><th>onehot_payment_type__5  </th><th>onehot_pu_hour_0  </th><th>onehot_pu_hour_1  </th><th>onehot_pu_hour_2  </th><th>onehot_pu_hour_3  </th><th>onehot_pu_hour_4  </th><th>onehot_pu_hour_5  </th><th>onehot_pu_hour_6  </th><th>onehot_pu_hour_7  </th><th>onehot_pu_hour_8  </th><th>onehot_pu_hour_9  </th><th>onehot_pu_hour_10  </th><th>onehot_pu_hour_11  </th><th>onehot_pu_hour_12  </th><th>onehot_pu_hour_13  </th><th>onehot_pu_hour_14  </th><th>onehot_pu_hour_15  </th><th>onehot_pu_hour_16  </th><th>onehot_pu_hour_17  </th><th>onehot_pu_hour_18  </th><th>onehot_pu_hour_19  </th><th>onehot_pu_hour_20  </th><th>onehot_pu_hour_21  </th><th>onehot_pu_hour_22  </th><th>onehot_pu_hour_23  </th><th>onehot_pu_day_of_week_0  </th><th>onehot_pu_day_of_week_1  </th><th>onehot_pu_day_of_week_2  </th><th>onehot_pu_day_of_week_3  </th><th>onehot_pu_day_of_week_4  </th><th>onehot_pu_day_of_week_5  </th><th>onehot_pu_day_of_week_6  </th><th>onehot_pu_month_0  </th><th>onehot_pu_month_1  </th><th>onehot_pu_month_2  </th><th>onehot_pu_month_3  </th><th>onehot_pu_month_4  </th><th>onehot_pu_month_5  </th><th>onehot_pu_month_6  </th><th>onehot_pu_month_7  </th><th>onehot_pu_month_8  </th><th>onehot_pu_month_9  </th><th>onehot_pu_month_10  </th><th>onehot_pu_month_11  </th><th>standard_scaled_arc_distance_miles  </th><th>standard_scaled_direction_angle  </th><th>standard_scaled_trip_distance  </th><th>linear_prediction  </th><th>final_prediction  </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i> </td><td>VTS        </td><td>2015-01-15 19:05:39.000000000</td><td>2015-01-15 19:23:42.000000000</td><td>1                </td><td>1             </td><td>1.590000033378601 </td><td>-73.993896484375  </td><td>40.7501106262207  </td><td>1.0        </td><td>0.0                 </td><td>-73.97478485107422 </td><td>40.75061798095703 </td><td>12.0         </td><td>1.0        </td><td>0.5      </td><td>3.25              </td><td>0.0              </td><td>17.049999237060547</td><td>5.285318670510585 </td><td>18.05              </td><td>7.5471696853637695</td><td>19       </td><td>3               </td><td>0         </td><td>0              </td><td>1.3205363750457764  </td><td>88.47933197021484  </td><td>0.01199185661971569   </td><td>-0.009015996009111404 </td><td>0.00014403648674488068</td><td>-0.0029410063289105892</td><td>-1             </td><td>10.341714928589358   </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                        </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                   </td><td>0                   </td><td>0.025801019743084908                </td><td>1.0132170915603638               </td><td>-0.37169644236564636           </td><td>10.070849907552892 </td><td>10.206282418071126</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i> </td><td>CMT        </td><td>2015-01-10 20:33:38.000000000</td><td>2015-01-10 20:53:28.000000000</td><td>1                </td><td>1             </td><td>3.299999952316284 </td><td>-74.00164794921875</td><td>40.7242431640625  </td><td>1.0        </td><td>0.0                 </td><td>-73.99441528320312 </td><td>40.75910949707031 </td><td>14.5         </td><td>0.5        </td><td>0.5      </td><td>2.0               </td><td>0.0              </td><td>17.799999237060547</td><td>9.983193133057666 </td><td>19.833333333333332 </td><td>4.39393949508667  </td><td>20       </td><td>5               </td><td>0         </td><td>1              </td><td>0.8311263918876648  </td><td>11.719215393066406 </td><td>0.035719096660614014  </td><td>0.0038769766688346863 </td><td>-0.005286338273435831 </td><td>0.017746448516845703  </td><td>-1             </td><td>15.396957926587826   </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                        </td><td>0                        </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                        </td><td>0                        </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                   </td><td>0                   </td><td>-0.35837456583976746                </td><td>0.26598840951919556              </td><td>0.5506355166435242             </td><td>13.105708647238803 </td><td>14.251333286913315</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i> </td><td>CMT        </td><td>2015-01-10 20:33:38.000000000</td><td>2015-01-10 20:43:41.000000000</td><td>1                </td><td>2             </td><td>1.7999999523162842</td><td>-73.96334075927734</td><td>40.80278778076172 </td><td>1.0        </td><td>0.0                 </td><td>-73.95182037353516 </td><td>40.82441329956055 </td><td>9.5          </td><td>0.5        </td><td>0.5      </td><td>0.0               </td><td>0.0              </td><td>10.800000190734863</td><td>10.746268372037516</td><td>10.05              </td><td>5.277778148651123 </td><td>20       </td><td>5               </td><td>0         </td><td>1              </td><td>0.8967181444168091  </td><td>28.045202255249023 </td><td>-0.046807631850242615 </td><td>-0.024863865226507187 </td><td>0.07238610833883286   </td><td>0.024521680548787117  </td><td>-1             </td><td>5.839834850469423    </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                        </td><td>0                        </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                        </td><td>0                        </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                   </td><td>0                   </td><td>-0.30688655376434326                </td><td>0.42491528391838074              </td><td>-0.25842761993408203           </td><td>7.854657477191768  </td><td>6.8472461638305955</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i> </td><td>CMT        </td><td>2015-01-10 20:33:39.000000000</td><td>2015-01-10 20:35:31.000000000</td><td>1                </td><td>2             </td><td>0.5               </td><td>-74.00908660888672</td><td>40.71381759643555 </td><td>1.0        </td><td>0.0                 </td><td>-74.00432586669922 </td><td>40.71998596191406 </td><td>3.5          </td><td>0.5        </td><td>0.5      </td><td>0.0               </td><td>0.0              </td><td>4.800000190734863 </td><td>16.071428571428573</td><td>1.8666666666666667 </td><td>7.0               </td><td>20       </td><td>5               </td><td>0         </td><td>1              </td><td>0.34927111864089966 </td><td>37.660945892333984 </td><td>0.04834355413913727   </td><td>0.006033187732100487  </td><td>-0.042189233005046844 </td><td>0.0014048703014850616 </td><td>-1             </td><td>3.9200899313720248   </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                        </td><td>0                        </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                        </td><td>0                        </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                   </td><td>0                   </td><td>-0.7366198897361755                 </td><td>0.5185206532478333               </td><td>-0.9596157073974609            </td><td>6.123777225634847  </td><td>5.021933578503436 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i> </td><td>CMT        </td><td>2015-01-10 20:33:39.000000000</td><td>2015-01-10 20:52:58.000000000</td><td>1                </td><td>2             </td><td>3.0               </td><td>-73.97117614746094</td><td>40.762428283691406</td><td>1.0        </td><td>0.0                 </td><td>-74.00418090820312 </td><td>40.742652893066406</td><td>15.0         </td><td>0.5        </td><td>0.5      </td><td>0.0               </td><td>0.0              </td><td>16.299999237060547</td><td>9.318377911993098 </td><td>19.316666666666666 </td><td>5.0               </td><td>20       </td><td>5               </td><td>0         </td><td>1              </td><td>2.311370849609375   </td><td>-120.92869567871094</td><td>-0.012809310108423233 </td><td>-0.0017468496225774288</td><td>-0.024262361228466034 </td><td>0.01527707651257515   </td><td>-1             </td><td>14.245256325838437   </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                        </td><td>0                        </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                        </td><td>0                        </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                   </td><td>0                   </td><td>0.8035832643508911                  </td><td>-1.025285243988037               </td><td>0.3888229429721832             </td><td>12.499606182582523 </td><td>13.372431254210479</td></tr>\n",
       "<tr><td>...                           </td><td>...        </td><td>...                          </td><td>...                          </td><td>...              </td><td>...           </td><td>...               </td><td>...               </td><td>...               </td><td>...        </td><td>...                 </td><td>...                </td><td>...               </td><td>...          </td><td>...        </td><td>...      </td><td>...               </td><td>...              </td><td>...               </td><td>...               </td><td>...                </td><td>...               </td><td>...      </td><td>...             </td><td>...       </td><td>...            </td><td>...                 </td><td>...                </td><td>...                   </td><td>...                   </td><td>...                   </td><td>...                   </td><td>...            </td><td>...                  </td><td>...                     </td><td>...                     </td><td>...                     </td><td>...                     </td><td>...                     </td><td>...               </td><td>...               </td><td>...               </td><td>...               </td><td>...               </td><td>...               </td><td>...               </td><td>...               </td><td>...               </td><td>...               </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                      </td><td>...                      </td><td>...                      </td><td>...                      </td><td>...                      </td><td>...                      </td><td>...                      </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                </td><td>...                 </td><td>...                 </td><td>...                                 </td><td>...                              </td><td>...                            </td><td>...                </td><td>...               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>16</i></td><td>VTS        </td><td>2015-01-15 19:05:40.000000000</td><td>2015-01-15 19:28:18.000000000</td><td>5                </td><td>1             </td><td>8.329999923706055 </td><td>-73.8630599975586 </td><td>40.76958084106445 </td><td>1.0        </td><td>0.0                 </td><td>-73.95271301269531 </td><td>40.78578186035156 </td><td>26.0         </td><td>1.0        </td><td>0.5      </td><td>8.079999923706055 </td><td>5.329999923706055</td><td>41.209999084472656</td><td>22.082474024552134</td><td>22.633333333333333 </td><td>3.121248483657837 </td><td>19       </td><td>3               </td><td>0         </td><td>0              </td><td>6.202259540557861   </td><td>-79.75674438476562 </td><td>-0.09456567466259003  </td><td>0.06935980916023254   </td><td>0.04143484681844711   </td><td>0.0013871975243091583 </td><td>-1             </td><td>18.416383849226644   </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                        </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                   </td><td>0                   </td><td>3.8578414916992188                  </td><td>-0.6244930028915405              </td><td>3.2636940479278564             </td><td>26.44849914215235  </td><td>22.432441495689496</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>17</i></td><td>VTS        </td><td>2015-01-15 19:05:41.000000000</td><td>2015-01-15 19:20:36.000000000</td><td>1                </td><td>1             </td><td>2.369999885559082 </td><td>-73.94554138183594</td><td>40.779422760009766</td><td>1.0        </td><td>0.0                 </td><td>-73.98085021972656 </td><td>40.78608322143555 </td><td>11.5         </td><td>1.0        </td><td>0.5      </td><td>0.0               </td><td>0.0              </td><td>13.300000190734863</td><td>9.532960433533738 </td><td>14.916666666666666 </td><td>4.852320671081543 </td><td>19       </td><td>3               </td><td>0         </td><td>0              </td><td>2.4429423809051514  </td><td>-79.31756591796875 </td><td>-0.04297391325235367  </td><td>0.004257339984178543  </td><td>0.024310370907187462  </td><td>0.02371533028781414   </td><td>-1             </td><td>13.417650454954236   </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                        </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                   </td><td>0                   </td><td>0.9068639278411865                  </td><td>-0.6202178001403809              </td><td>0.04901633784174919            </td><td>11.301022815214012 </td><td>12.359336635084123</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>18</i></td><td>VTS        </td><td>2015-01-15 19:05:41.000000000</td><td>2015-01-15 19:20:22.000000000</td><td>2                </td><td>1             </td><td>7.130000114440918 </td><td>-73.87445831298828</td><td>40.774009704589844</td><td>1.0        </td><td>0.0                 </td><td>-73.95237731933594 </td><td>40.718589782714844</td><td>21.5         </td><td>1.0        </td><td>0.5      </td><td>4.5               </td><td>0.0              </td><td>27.799999237060547</td><td>29.135074247431675</td><td>14.683333333333334 </td><td>3.015427827835083 </td><td>19       </td><td>3               </td><td>0         </td><td>0              </td><td>5.487300395965576   </td><td>-125.42235565185547</td><td>-0.08959843218326569  </td><td>0.058185599744319916  </td><td>-0.011233885772526264 </td><td>-0.04033680260181427  </td><td>-1             </td><td>17.78153347481701    </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                        </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                   </td><td>0                   </td><td>3.2966151237487793                  </td><td>-1.0690292119979858              </td><td>2.616443634033203              </td><td>24.032145199466143 </td><td>20.906839337141577</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>19</i></td><td>VTS        </td><td>2015-01-15 19:05:41.000000000</td><td>2015-01-15 19:31:00.000000000</td><td>1                </td><td>2             </td><td>3.5999999046325684</td><td>-73.97660064697266</td><td>40.751895904541016</td><td>1.0        </td><td>0.0                 </td><td>-73.9989242553711  </td><td>40.714595794677734</td><td>17.5         </td><td>1.0        </td><td>0.5      </td><td>0.0               </td><td>0.0              </td><td>19.299999237060547</td><td>8.531928674573566 </td><td>25.316666666666666 </td><td>4.861111164093018 </td><td>19       </td><td>3               </td><td>0         </td><td>0              </td><td>1.698374629020691   </td><td>-149.1000518798828 </td><td>-0.0015387892490252852</td><td>0.0019043900538235903 </td><td>-0.04309796541929245  </td><td>-0.006171772256493568 </td><td>-1             </td><td>16.72275687455656    </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                        </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                   </td><td>0                   </td><td>0.32239535450935364                 </td><td>-1.299522042274475               </td><td>0.7124481201171875             </td><td>15.722191611514923 </td><td>16.22247424303574 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>20</i></td><td>VTS        </td><td>2015-01-15 19:05:41.000000000</td><td>2015-01-15 19:10:22.000000000</td><td>1                </td><td>1             </td><td>0.8899999856948853</td><td>-73.99495697021484</td><td>40.745079040527344</td><td>1.0        </td><td>0.0                 </td><td>-73.99993896484375 </td><td>40.734649658203125</td><td>5.5          </td><td>1.0        </td><td>0.5      </td><td>1.6200000047683716</td><td>0.0              </td><td>8.920000076293945 </td><td>11.402135048048352</td><td>4.683333333333334  </td><td>6.179775238037109 </td><td>19       </td><td>3               </td><td>0         </td><td>0              </td><td>0.39743757247924805 </td><td>-154.46673583984375</td><td>0.016289759427309036  </td><td>-0.0061929612420499325</td><td>-0.027942970395088196 </td><td>0.007000666111707687  </td><td>-1             </td><td>5.2916857480635      </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                       </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                 </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                        </td><td>0                        </td><td>0                        </td><td>0                        </td><td>1                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                   </td><td>0                   </td><td>-0.6988102793693542                 </td><td>-1.3517645597457886              </td><td>-0.7492592930793762            </td><td>8.109655135730126  </td><td>6.700670441896813 </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#    vendor_id    pickup_datetime                dropoff_datetime               passenger_count    payment_type    trip_distance       pickup_longitude    pickup_latitude     rate_code    store_and_fwd_flag    dropoff_longitude    dropoff_latitude    fare_amount    surcharge    mta_tax    tip_amount          tolls_amount       total_amount        trip_speed_mph      trip_duration_min    fare_by_distance    pu_hour    pu_day_of_week    pu_month    pu_is_weekend    arc_distance_miles    direction_angle      PCA_0                   PCA_1                   PCA_2                   PCA_3                   payment_type_    lightgbm_prediction    onehot_payment_type__1    onehot_payment_type__2    onehot_payment_type__3    onehot_payment_type__4    onehot_payment_type__5    onehot_pu_hour_0    onehot_pu_hour_1    onehot_pu_hour_2    onehot_pu_hour_3    onehot_pu_hour_4    onehot_pu_hour_5    onehot_pu_hour_6    onehot_pu_hour_7    onehot_pu_hour_8    onehot_pu_hour_9    onehot_pu_hour_10    onehot_pu_hour_11    onehot_pu_hour_12    onehot_pu_hour_13    onehot_pu_hour_14    onehot_pu_hour_15    onehot_pu_hour_16    onehot_pu_hour_17    onehot_pu_hour_18    onehot_pu_hour_19    onehot_pu_hour_20    onehot_pu_hour_21    onehot_pu_hour_22    onehot_pu_hour_23    onehot_pu_day_of_week_0    onehot_pu_day_of_week_1    onehot_pu_day_of_week_2    onehot_pu_day_of_week_3    onehot_pu_day_of_week_4    onehot_pu_day_of_week_5    onehot_pu_day_of_week_6    onehot_pu_month_0    onehot_pu_month_1    onehot_pu_month_2    onehot_pu_month_3    onehot_pu_month_4    onehot_pu_month_5    onehot_pu_month_6    onehot_pu_month_7    onehot_pu_month_8    onehot_pu_month_9    onehot_pu_month_10    onehot_pu_month_11    standard_scaled_arc_distance_miles    standard_scaled_direction_angle    standard_scaled_trip_distance    linear_prediction    final_prediction\n",
       "0    VTS          2015-01-15 19:05:39.000000000  2015-01-15 19:23:42.000000000  1                  1               1.590000033378601   -73.993896484375    40.7501106262207    1.0          0.0                   -73.97478485107422   40.75061798095703   12.0           1.0          0.5        3.25                0.0                17.049999237060547  5.285318670510585   18.05                7.5471696853637695  19         3                 0           0                1.3205363750457764    88.47933197021484    0.01199185661971569     -0.009015996009111404   0.00014403648674488068  -0.0029410063289105892  -1               10.341714928589358     0                         0                         0                         0                         0                         0                   0                   0                   0                   0                   0                   0                   0                   0                   0                   0                    0                    0                    0                    0                    0                    0                    0                    0                    1                    0                    0                    0                    0                    0                          0                          0                          1                          0                          0                          0                          1                    0                    0                    0                    0                    0                    0                    0                    0                    0                    0                     0                     0.025801019743084908                  1.0132170915603638                 -0.37169644236564636             10.070849907552892   10.206282418071126\n",
       "1    CMT          2015-01-10 20:33:38.000000000  2015-01-10 20:53:28.000000000  1                  1               3.299999952316284   -74.00164794921875  40.7242431640625    1.0          0.0                   -73.99441528320312   40.75910949707031   14.5           0.5          0.5        2.0                 0.0                17.799999237060547  9.983193133057666   19.833333333333332   4.39393949508667    20         5                 0           1                0.8311263918876648    11.719215393066406   0.035719096660614014    0.0038769766688346863   -0.005286338273435831   0.017746448516845703    -1               15.396957926587826     0                         0                         0                         0                         0                         0                   0                   0                   0                   0                   0                   0                   0                   0                   0                   0                    0                    0                    0                    0                    0                    0                    0                    0                    0                    1                    0                    0                    0                    0                          0                          0                          0                          0                          1                          0                          1                    0                    0                    0                    0                    0                    0                    0                    0                    0                    0                     0                     -0.35837456583976746                  0.26598840951919556                0.5506355166435242               13.105708647238803   14.251333286913315\n",
       "2    CMT          2015-01-10 20:33:38.000000000  2015-01-10 20:43:41.000000000  1                  2               1.7999999523162842  -73.96334075927734  40.80278778076172   1.0          0.0                   -73.95182037353516   40.82441329956055   9.5            0.5          0.5        0.0                 0.0                10.800000190734863  10.746268372037516  10.05                5.277778148651123   20         5                 0           1                0.8967181444168091    28.045202255249023   -0.046807631850242615   -0.024863865226507187   0.07238610833883286     0.024521680548787117    -1               5.839834850469423      0                         0                         0                         0                         0                         0                   0                   0                   0                   0                   0                   0                   0                   0                   0                   0                    0                    0                    0                    0                    0                    0                    0                    0                    0                    1                    0                    0                    0                    0                          0                          0                          0                          0                          1                          0                          1                    0                    0                    0                    0                    0                    0                    0                    0                    0                    0                     0                     -0.30688655376434326                  0.42491528391838074                -0.25842761993408203             7.854657477191768    6.8472461638305955\n",
       "3    CMT          2015-01-10 20:33:39.000000000  2015-01-10 20:35:31.000000000  1                  2               0.5                 -74.00908660888672  40.71381759643555   1.0          0.0                   -74.00432586669922   40.71998596191406   3.5            0.5          0.5        0.0                 0.0                4.800000190734863   16.071428571428573  1.8666666666666667   7.0                 20         5                 0           1                0.34927111864089966   37.660945892333984   0.04834355413913727     0.006033187732100487    -0.042189233005046844   0.0014048703014850616   -1               3.9200899313720248     0                         0                         0                         0                         0                         0                   0                   0                   0                   0                   0                   0                   0                   0                   0                   0                    0                    0                    0                    0                    0                    0                    0                    0                    0                    1                    0                    0                    0                    0                          0                          0                          0                          0                          1                          0                          1                    0                    0                    0                    0                    0                    0                    0                    0                    0                    0                     0                     -0.7366198897361755                   0.5185206532478333                 -0.9596157073974609              6.123777225634847    5.021933578503436\n",
       "4    CMT          2015-01-10 20:33:39.000000000  2015-01-10 20:52:58.000000000  1                  2               3.0                 -73.97117614746094  40.762428283691406  1.0          0.0                   -74.00418090820312   40.742652893066406  15.0           0.5          0.5        0.0                 0.0                16.299999237060547  9.318377911993098   19.316666666666666   5.0                 20         5                 0           1                2.311370849609375     -120.92869567871094  -0.012809310108423233   -0.0017468496225774288  -0.024262361228466034   0.01527707651257515     -1               14.245256325838437     0                         0                         0                         0                         0                         0                   0                   0                   0                   0                   0                   0                   0                   0                   0                   0                    0                    0                    0                    0                    0                    0                    0                    0                    0                    1                    0                    0                    0                    0                          0                          0                          0                          0                          1                          0                          1                    0                    0                    0                    0                    0                    0                    0                    0                    0                    0                     0                     0.8035832643508911                    -1.025285243988037                 0.3888229429721832               12.499606182582523   13.372431254210479\n",
       "...  ...          ...                            ...                            ...                ...             ...                 ...                 ...                 ...          ...                   ...                  ...                 ...            ...          ...        ...                 ...                ...                 ...                 ...                  ...                 ...        ...               ...         ...              ...                   ...                  ...                     ...                     ...                     ...                     ...              ...                    ...                       ...                       ...                       ...                       ...                       ...                 ...                 ...                 ...                 ...                 ...                 ...                 ...                 ...                 ...                 ...                  ...                  ...                  ...                  ...                  ...                  ...                  ...                  ...                  ...                  ...                  ...                  ...                  ...                  ...                        ...                        ...                        ...                        ...                        ...                        ...                        ...                  ...                  ...                  ...                  ...                  ...                  ...                  ...                  ...                  ...                  ...                   ...                   ...                                   ...                                ...                              ...                  ...\n",
       "16   VTS          2015-01-15 19:05:40.000000000  2015-01-15 19:28:18.000000000  5                  1               8.329999923706055   -73.8630599975586   40.76958084106445   1.0          0.0                   -73.95271301269531   40.78578186035156   26.0           1.0          0.5        8.079999923706055   5.329999923706055  41.209999084472656  22.082474024552134  22.633333333333333   3.121248483657837   19         3                 0           0                6.202259540557861     -79.75674438476562   -0.09456567466259003    0.06935980916023254     0.04143484681844711     0.0013871975243091583   -1               18.416383849226644     0                         0                         0                         0                         0                         0                   0                   0                   0                   0                   0                   0                   0                   0                   0                   0                    0                    0                    0                    0                    0                    0                    0                    0                    1                    0                    0                    0                    0                    0                          0                          0                          1                          0                          0                          0                          1                    0                    0                    0                    0                    0                    0                    0                    0                    0                    0                     0                     3.8578414916992188                    -0.6244930028915405                3.2636940479278564               26.44849914215235    22.432441495689496\n",
       "17   VTS          2015-01-15 19:05:41.000000000  2015-01-15 19:20:36.000000000  1                  1               2.369999885559082   -73.94554138183594  40.779422760009766  1.0          0.0                   -73.98085021972656   40.78608322143555   11.5           1.0          0.5        0.0                 0.0                13.300000190734863  9.532960433533738   14.916666666666666   4.852320671081543   19         3                 0           0                2.4429423809051514    -79.31756591796875   -0.04297391325235367    0.004257339984178543    0.024310370907187462    0.02371533028781414     -1               13.417650454954236     0                         0                         0                         0                         0                         0                   0                   0                   0                   0                   0                   0                   0                   0                   0                   0                    0                    0                    0                    0                    0                    0                    0                    0                    1                    0                    0                    0                    0                    0                          0                          0                          1                          0                          0                          0                          1                    0                    0                    0                    0                    0                    0                    0                    0                    0                    0                     0                     0.9068639278411865                    -0.6202178001403809                0.04901633784174919              11.301022815214012   12.359336635084123\n",
       "18   VTS          2015-01-15 19:05:41.000000000  2015-01-15 19:20:22.000000000  2                  1               7.130000114440918   -73.87445831298828  40.774009704589844  1.0          0.0                   -73.95237731933594   40.718589782714844  21.5           1.0          0.5        4.5                 0.0                27.799999237060547  29.135074247431675  14.683333333333334   3.015427827835083   19         3                 0           0                5.487300395965576     -125.42235565185547  -0.08959843218326569    0.058185599744319916    -0.011233885772526264   -0.04033680260181427    -1               17.78153347481701      0                         0                         0                         0                         0                         0                   0                   0                   0                   0                   0                   0                   0                   0                   0                   0                    0                    0                    0                    0                    0                    0                    0                    0                    1                    0                    0                    0                    0                    0                          0                          0                          1                          0                          0                          0                          1                    0                    0                    0                    0                    0                    0                    0                    0                    0                    0                     0                     3.2966151237487793                    -1.0690292119979858                2.616443634033203                24.032145199466143   20.906839337141577\n",
       "19   VTS          2015-01-15 19:05:41.000000000  2015-01-15 19:31:00.000000000  1                  2               3.5999999046325684  -73.97660064697266  40.751895904541016  1.0          0.0                   -73.9989242553711    40.714595794677734  17.5           1.0          0.5        0.0                 0.0                19.299999237060547  8.531928674573566   25.316666666666666   4.861111164093018   19         3                 0           0                1.698374629020691     -149.1000518798828   -0.0015387892490252852  0.0019043900538235903   -0.04309796541929245    -0.006171772256493568   -1               16.72275687455656      0                         0                         0                         0                         0                         0                   0                   0                   0                   0                   0                   0                   0                   0                   0                   0                    0                    0                    0                    0                    0                    0                    0                    0                    1                    0                    0                    0                    0                    0                          0                          0                          1                          0                          0                          0                          1                    0                    0                    0                    0                    0                    0                    0                    0                    0                    0                     0                     0.32239535450935364                   -1.299522042274475                 0.7124481201171875               15.722191611514923   16.22247424303574\n",
       "20   VTS          2015-01-15 19:05:41.000000000  2015-01-15 19:10:22.000000000  1                  1               0.8899999856948853  -73.99495697021484  40.745079040527344  1.0          0.0                   -73.99993896484375   40.734649658203125  5.5            1.0          0.5        1.6200000047683716  0.0                8.920000076293945   11.402135048048352  4.683333333333334    6.179775238037109   19         3                 0           0                0.39743757247924805   -154.46673583984375  0.016289759427309036    -0.0061929612420499325  -0.027942970395088196   0.007000666111707687    -1               5.2916857480635        0                         0                         0                         0                         0                         0                   0                   0                   0                   0                   0                   0                   0                   0                   0                   0                    0                    0                    0                    0                    0                    0                    0                    0                    1                    0                    0                    0                    0                    0                          0                          0                          1                          0                          0                          0                          1                    0                    0                    0                    0                    0                    0                    0                    0                    0                    0                     0                     -0.6988102793693542                   -1.3517645597457886                -0.7492592930793762              8.109655135730126    6.700670441896813"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Lets see what happened\n",
    "df_test.head(21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-10T20:42:09.574540Z",
     "start_time": "2019-07-10T20:42:03.289076Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The mean absolute error is 2.713\n",
      "The mean squared score is 17.956\n"
     ]
    }
   ],
   "source": [
    "# Check the performance\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error\n",
    "\n",
    "mae_test_score = mean_absolute_error(df_test[:1_000_000].trip_duration_min.values, \n",
    "                                     df_test[:1_000_000].final_prediction.values)\n",
    "mse_test_score = mean_squared_error(df_test[:1_000_000].trip_duration_min.values, \n",
    "                                    df_test[:1_000_000].final_prediction.values)\n",
    "\n",
    "print('The mean absolute error is %2.3f' % mae_test_score)\n",
    "print('The mean squared score is %2.3f' % mse_test_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Thank you for your attention!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
